__author__ = "Henry Heberle"
__date__ = "$25/10/2010 20:34:29$"

from decimal import *
import random
import sys
from time import gmtime
from time import strftime

import matplotlib.pyplot as plt
import nltk
from nltk import stem
from nltk.corpus import stopwords
from nltk.probability import *
import pg
import vtk
from vtk import *

if __name__ == "__main__":

    punctuations = [']', '[', '.', ',', ';', '"', '\'', '?', '(', ')', ':', '-',
    '_', '`', "#", "&", "|", "--", "&quot", "&ndash", "&mdash",
    "/prnewswire-usnewswire/", "'s", "u'"]

    stopwords = nltk.corpus.stopwords.words('english')
    notwords = punctuations + stopwords
    #print "stopwords:"
    #print stopwords

    tempo = []
    #tempo.append(strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()))
    print strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())


    def similarity(textoI, textoJ):
        I = set(textoI)
        J = set(textoJ)
        if textoI == textoJ:
            return 0
        try:
            A = len(I | J)   #c<=5 -> A/2B <= 5 -> A/B <= 10 -> 100/10 ou 50/5 ou 25/2,5 ou 10/1
            B = len(I & J)
            #print "Tamanho: "+ str(A) +"/"+str(B)
            co = A / (B * 2)
        except ZeroDivisionError:
            return -1
        return co

    def getcategory(number):
        if number == 0:
            return "technology"
        elif number == 1:
            return "entertainment and arts"
        elif number == 2:
            return "health"
        elif number == 3:
            return"business"
        elif number == 4:
            return "sports"
        elif number == 5:
            return "science"
        elif number == 6:
            return "politics"

    all_words = []
    fwc = {} #fdist_words_category

    #Leitura do feed de noticias
    #Transforma cada noticia em uma lista de palavras
    #Armazena a lista i em words_text[i]
    words_text = []
    words_text_list = []
    titleList = []
    summaryList = []
    classList = []
    classListNumeric = []
    wnl = stem.WordNetLemmatizer()

    db = pg.connect("rssdb", "localhost", 5432, None, None, "rssuser", "password")

    for item in db.query("SELECT title, summary, category FROM RSS").getresult(): #LIMIT 2000
        try:
            title = nltk.clean_html(item[0])[2:-1].encode("ascii")
            summary = nltk.clean_html(item[1])[2:-1].encode("ascii")
            text = title + " " + summary
            key = item[2]
            #print "::: "+key+" ::: "+text
            #transforma o texto em lista
            words = [w.lower() for w in nltk.word_tokenize(text) if len(w) > 1]

            #trata algumas palavras por conta de unicode e
            #ponto final em final de texto
            for i in range(len(words)):                    
                words[i].replace(".", " ")

            #remove palavras com 1 caractere
            for wd in words:
                if len(wd) < 2:
                    words.remove(wd)

            #remove stopwords
            words = [w.lower() for w in words if w not in notwords]

            if sys.argv[1] == "porter":
                #Stemmers
                porter = nltk.PorterStemmer()
                porterWords = [porter.stem(w) for w in words]
                processedWords = porterWords
            elif sys.argv[1] == "lancaster":
                lancaster = nltk.LancasterStemmer()
                lancasterWords = [lancaster.stem(w) for w in words]
                processedWords = lancasterWords
            elif sys.argv[1] =="lemmatizer":
                #lemas. ex: loves se torna: love
                lemmatizedWords = [wnl.lemmatize(w) for w in words]
                processedWords = lemmatizedWords

            #elimina palavras repetidas
            listSet = list(set(processedWords))

            #print text
            #print listSet
            #print lemmatizedWords
            #raw_input()

            words_text.append(listSet)
            words_text_list.append(processedWords)
            #listas de titulos e resumos das noticias, mapeada para cada vertice i
            titleList.append(title)
            summaryList.append(summary)

            #listas de categorias, mapeadas para cada vertice i
            classList.append(getcategory(key)) #string da categoria
            classListNumeric.append(key) #numero  da categoria
        except Exception as e:
            print e
            pass
    db.close()

    #Diminui o espaco de palavras
    #words_text = words_text[:300] #mudar no SQL

    weight = []
    number_of_texts = len(words_text)
    keys_text = range(number_of_texts)

    def getConstPerCent(percent):
        """Retorna uma constante que representa a distancia entre as noticias;
                              quao diferentes sao. 0.5 significa muito similar"""
        return 100 / (2 * percent)
    #similarity: % -> Const
    #100 = 0.50		40 = 1.25
    #90  = 0.56		30 = 1.667
    #80  = 0.63		20 = 2.5
    #70  = 0.72		10 = 5.0
    #60  = 0.84		05 = 10.0
    #50  = 1.00

    edgesListaux = [] #list of edges. some of them will be removed
    weightListaux = []
    max = 99999999
    constante = getConstPerCent(15)
    for i in keys_text:
        l = []
        for j in keys_text:
            co = similarity(words_text[i], words_text[j])
            if co == -1:
                co = max
            l.append(co)
            if co < constante:
                if i != j and [j, i] not in edgesListaux and [i, j] not in edgesListaux:
                    edgesListaux.append([i, j])
                    weightListaux.append(co)
        weight.append(l)

    print "Calculou pesos para as arestas, geral, do grafo"
    print strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())

    def printNews(i, j):
        print titleList[i] + "\n" + summaryList[i] + "\n"
        print titleList[j] + "\n" + summaryList[j]
        print "--------------------------------------------------------------"

    def getProbability(edgs):
        used = []
        prob = []
        for v in edgs:
            i = v[0]
            j = v[1]
            if i != j and [i, j] not in used and [j, i] not in used:
                used.append([i, j])
                printNews(i, j)
                c = similarity(words_text[i], words_text[j])
                simil = raw_input()
                prob.append([[i, j], c, simil])
        return prob

    def saveProb(prob):
        file = open("prob.txt", "w")
        for p in prob:
            edg = p[0]
            c = p[1]
            simil = p[2]
            line = str(edg[0]) + " " + str(edg[1]) + " " + str(c) + " " + str(simil) + "\n"
            file.write(line)
        file.close()

    def selectEdges(num):
        edg1, edg2, edg3, edg4 = [], [], [], []
        i = 0
        for l in weight:
            j = 0
            for c in l:
                if c < 1.0 and c >= 0.5:
                    edg1.append([i, j])
                elif c < 1.5 and c >= 1.0:
                    edg2.append([i, j])
                elif c < 2.0 and c >= 1.5:
                    edg3.append([i, j])
                elif c < 2.5 and c >= 2.0:
                    edg4.append([i, j])
                j+=1
            i+=1
        random.shuffle(edg1)
        random.shuffle(edg2)
        random.shuffle(edg4)
        random.shuffle(edg1)
        random.shuffle(edg3)
        random.shuffle(edg3)
        random.shuffle(edg4)
        random.shuffle(edg2)
        print edg1
        print edg2
        print edg3
        print edg4
        edg = edg1[:num] + edg2[:num] + edg3[:num] + edg4[:num]
        return edg

    #selectedEdges  = selectEdges(3)
    #print selectedEdges
    #prob = getProbability(selectedEdges)
    #saveProb(prob)
    """
    x, y = [], []
    for obj in prob:
        x.append(obj[1])
        y.append(obj[2])
    plt.plot(x, y)
    plt.axis([0, 3, 0, 4])
    plt.show()
    """
    vertex = []
    g = vtk.vtkMutableDirectedGraph()
    vertices = vtk.vtkDoubleArray()
    vertices.SetNumberOfComponents(1)
    vertices.SetName("Vertices")

    #Create N vertices
    for k in range(number_of_texts):
        vertex.append(g.AddVertex())
        vertices.InsertNextValue(k)


    colors_v = vtk.vtkDoubleArray()
    colors_v.SetNumberOfComponents(1)
    colors_v.SetName("color")
    #colors for N vertices
    for k in range(number_of_texts):
        colors_v.InsertNextValue(classListNumeric[k])

    #Colors for edges that will be created
    colors_e = vtk.vtkDoubleArray()
    colors_e.SetNumberOfComponents(1)
    colors_e.SetName("color")

    total = 0
    #auxiliar lists
    weightList = []

    #weights of edges of the vtkGraph
    weights = vtk.vtkDoubleArray()
    weights.SetNumberOfComponents(1)
    weights.SetName("Weights")

    """
    i = 0
    constante = getConstPerCent(5)
    #Select only some edges to create. Big distances(diferent texts) will not be created.
    for l in weight:
        j = 0
        for c in l:
            if c < constante:
                if i != j and [j, i] not in edgesListaux and [i, j] not in edgesListaux:
                    a = [i, j]
                    edgesListaux.append(a)
                    weightListaux.append(c)
            j += 1
        i += 1
     """

    #auxiliar list to treate (i,j), (i,i) and (j,i) edges like one.
    edgesList = []

    #number of edges created in the last for-loop
    numberedges = range(len(edgesListaux))
    motivoEdge = []
    porsozinho = 0
    porvizinho = 0
    porotimo = 0
    portextocurto = 0

    #return a color value for a weight gived, const
    def getColorEdge(weightE):
        if weightE < 1.25:
            return 5
        elif weightE <= 1.60:
            return 4
        elif weightE <= 2.19:
            return 3
        elif weightE <= 2.7:
            return 2
        elif weightE <= 5.0:
            return 1
        else:
            return 0

    #edges that will really be considered to be drawn
    resultEdges = []


    #Euristics to draw the graph
    for k in numberedges:
        edge = edgesListaux[k]
        v0 = edge[0]
        v1 = edge[1]
        if v0 != v1 and [v1, v0] not in edgesList and [v0, v1] not in edgesList:
        #Se nao for uma aresta repetida
            C = getConstPerCent(40)
            if weightListaux[k] >= C: #se os textos forem distantes, nao importa o tamanho deles
                Co = getConstPerCent(1)

                #seleciona vertices que possuem arestas, ou seja, cujo peso eh maior que uma constante C
                #de um loop-for anterior
                vList0 = [edg[1] for edg in edgesListaux if edg[0] == v0] #vertices vizinhos
                vList0 = vList0 + [edg[0] for edg in edgesListaux if edg[1] == v0] #vertices vizinhos
                vList0 = list(set(vList0))
                newVList0 = []
                for e in vList0:
                    if [e, v0] not in newVList0 and [v0, e] not in newVList0:
                        newVList0.append(e)

                vList1 = [edg[1] for edg in edgesListaux if edg[0] == v1]
                vList1 = vList1 + [edg[0] for edg in edgesListaux if edg[1] == v1]
                vList1 = list(set(vList1))
                newVList1 = []
                for e in vList1:
                    if [e, v1] not in newVList0 and [v1, e] not in newVList0:
                        newVList1.append(e)

                text0 = words_text[v0]
                for v in newVList0:
                    text0 = text0 + words_text[v]

                text1 = words_text[v1]
                for v in newVList1:
                    text1 = text1 + words_text[v]


                #calcular a similaridade entre o grupo de vizinhos de cada vertice e saber se a
                #aresta deve ou nao ser desenhada
                if len(words_text[v0]) > 15 and len(words_text[v1]) > 15:
                    const = similarity(text0, text1)
                    Cgrupo = getConstPerCent(25)
                    #print "entrou em vizinhanca!"
                    if const <= Cgrupo:
                        #print "similaridade de grupo boa, adiciona aresta"
                        g.AddGraphEdge(v0, v1)
                        weights.InsertNextValue(weightListaux[k] / 1.0)
                        weightList.append(weightListaux[k] / 1.0)
                        colors_e.InsertNextValue(getColorEdge(weightListaux[k]))
                        edgesList.append([v0, v1])
                        motivoEdge.append("similaridade de grupo, vizinhos")
                        porvizinho += 1
            #se o peso da aresta for pequeno, nao consideramos os vizinhos
            #para peso pequeno o tamanho do texto importa
            else:
                C2 = getConstPerCent(60)
                if(weightListaux[k] > C2):
                    if len(words_text[v0]) < 20 and len(words_text[v1]) < 20:
                        print "textos potencialmente ruins"
                    else:
                        #print "boa precisao! adicionou direto"
                        g.AddGraphEdge(v0, v1)
                        weights.InsertNextValue(weightListaux[k] / 1.0)
                        weightList.append(weightListaux[k] / 1.0)
                        colors_e.InsertNextValue(getColorEdge(weightListaux[k]))
                        edgesList.append([v0, v1])
                        motivoEdge.append("precisao <" + str(C) + " e >" + str(C2) + ", tamanhos dos texto1 e 2 < 20")
                        portextocurto += 1
                else:
                    #print "boa precisao! adicionou direto"
                    g.AddGraphEdge(v0, v1)
                    weights.InsertNextValue(weightListaux[k] / 1.0)
                    weightList.append(weightListaux[k] / 1.0)
                    colors_e.InsertNextValue(getColorEdge(weightListaux[k]))
                    edgesList.append([v0, v1])
                    motivoEdge.append("precisao < " + str(C2))
                    porotimo += 1

    print "Calculou arestas de vizinhos e precisao direta"
    print strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())

    #Para cada vertice que nao foi ligado a outro por nao possuir grande similaridade
    #crie uma aresta com a melhor similaridade possivel; menor coeficiente possivel.
    aloneVList = []
    notAloneVList = []
    aloneVList = vertex[:]
    notAloneVList = vertex[:]

    for e in edgesList:
        try:
            aloneVList.remove(e[0])
        except:
            try:
                aloneVList.remove(e[1])
            except:
                pass

    for v in aloneVList:
        notAloneVList.remove(v)

    #Retorna a aresta com menor coeficiente possivel para o vertice v0
    def getBetterSimilarityEdge(v0):
        better_w = 5000000
        better_v = 0
        #considerar apenas arestas de distancia menor que c
        c = getConstPerCent(5)
        #if(len(notAloneVList)> 0):
        for v1 in vertex:
            if v1 != v0:
                w = similarity(words_text[v0], words_text[v1])
                if w == -1 or w < 2:
                    w = max
                if w < better_w:
                    better_w = w
                    better_v = v1
        if better_w > c:
            return -1
        else:
            return better_v
        #else:
        #    return -1

    #Cria a aresta com menor coeficiente possivel
    for v0 in aloneVList:
        v1 = getBetterSimilarityEdge(v0)
        if v1 != -1 and v0 != v1 and [v0, v1] not in edgesList and [v1, v0] not in edgesList:
            g.AddGraphEdge(v0, v1)
            #O peso nestes casos nao importam e as arestas devem ser quase invisiveis
            weights.InsertNextValue(max)
            weightList.append(max)
            colors_e.InsertNextValue(getColorEdge(max))
            edgesList.append([v0, v1])
            motivoEdge.append("vertice sozinho")
            porsozinho += 1
            aloneVList.remove(v0)
            notAloneVList.append(v0)

    print "Calculou arestas de vertices sozinhos"
    print strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())

    print "\nIniciando rendering..."
    # Add the edge weight array to the graph
    g.GetEdgeData().AddArray(weights)
    g.GetEdgeData().AddArray(colors_e)

    g.GetVertexData().AddArray(vertices)
    g.GetVertexData().AddArray(colors_v)

    graphLayoutView = vtk.vtkGraphLayoutView()

    if sys.argv[2] == "2D3D":
        strategy = vtkForceDirectedLayoutStrategy()
        strategy.SetInitialTemperature(0)
        #strategy.AutomaticBoundsComputationOn()
        graphLayoutView.SetLayoutStrategy(strategy)
        graphLayoutView.SetInteractionModeTo3D()
    elif sys.argv[2] == "2D":
        strategy = vtkForceDirectedLayoutStrategy()
        strategy.SetInitialTemperature(0)
        graphLayoutView.SetLayoutStrategy(strategy)
    elif sys.argv[2] == "5":
        strategy = vtkForceDirectedLayoutStrategy()
        strategy.SetInitialTemperature(5)
        strategy.AutomaticBoundsComputationOn()
        graphLayoutView.SetLayoutStrategy(strategy)
        graphLayoutView.SetInteractionModeTo3D()
    elif sys.argv[2] == "3D":
        strategy = vtkForceDirectedLayoutStrategy()
        strategy.SetThreeDimensionalLayout(1)
        strategy.SetInitialTemperature(0)
        strategy.AutomaticBoundsComputationOn()
        graphLayoutView.SetLayoutStrategy(strategy)
        graphLayoutView.SetInteractionModeTo3D()
    else:
        strategy = vtkForceDirectedLayoutStrategy()
        #strategy.SetThreeDimensionalLayout(1)
        strategy.SetInitialTemperature(5)
        strategy.SetAutomaticBoundsComputation(1)
        #strategy.AutomaticBoundsComputationOn()
        graphLayoutView.SetLayoutStrategy(strategy)
        graphLayoutView.SetInteractionModeTo3D()

    graphLayoutView.AddRepresentationFromInput(g)


    theme = vtkViewTheme.CreateMellowTheme()
    theme.SetPointSize(4)
    theme.SetLineWidth(1)
    theme.SetCellOpacity(.99)
    theme.SetSelectedCellColor(1, 0, 1)
    theme.SetSelectedPointColor(1, 0, 1)
    theme.SetBackgroundColor(0.2, 0.2, 0.2)
    theme.SetBackgroundColor2(0.1, 0.1, 0.1)
    graphLayoutView.ApplyViewTheme(theme)
    graphLayoutView.SetVertexLabelFontSize(4)
    theme.FastDelete()


    #graphLayoutView.SetLayoutStrategy("Simple 2D")
    graphLayoutView.SetEdgeLabelArrayName("Weights")
    #graphLayoutView.SetEdgeLabelVisibility(1)
    graphLayoutView.SetEdgeColorArrayName("color")

    graphLayoutView.SetColorEdges(True)

    graphLayoutView.SetVertexLabelArrayName("Vertices")
    #graphLayoutView.SetVertexLabelVisibility(1)
    graphLayoutView.SetVertexColorArrayName("color")
    graphLayoutView.SetColorVertices(True)

    #graphLayoutView.GetLayoutStrategy().SetEdgeWeightField("Weights")
    #graphLayoutView.GetLayoutStrategy().SetWeightEdges(1)

    #tempo.append(strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()))
    #print tempo
    graphLayoutView.ResetCamera()
    graphLayoutView.Render()

    print "Terminou rendering."
    print strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())

    #graphLayoutView.GetLayoutStrategy().SetRandomSeed(0)

    #words_text_list[t]
    def getIntersectionTexts(index):
        #palavras que aparecem em todos os textos
        inter = set(words_text[index[0]])
        for t in index:
            inter = set(words_text[t]) & inter
        return inter

    def selectionCallback(caller, event):
        # In C++ there is some extra data passed to the callback, but in Python
        # the callback data is lost...

        # There should be two selection nodes, but which one is vertices and which is edges
        # does not seem to be guaranteed...
        sel = caller.GetCurrentSelection()
        node0 = sel.GetNode(0)
        node0_field_type = node0.GetFieldType()
        sel_list0 = caller.GetCurrentSelection().GetNode(0).GetSelectionList()
        node1 = sel.GetNode(1)
        node1_field_type = node1.GetFieldType()
        sel_list1 = caller.GetCurrentSelection().GetNode(1).GetSelectionList()

        if (sel_list0.GetNumberOfTuples() > 0):
            printFieldType(node0_field_type)
            if node0_field_type == 3:
                if(sel_list0.GetNumberOfTuples() == 1):
                    x = sel_list0.GetValue(0)
                    edlist = [e for e in edgesList if weightList[edgesList.index(e)] < 2.2]
                    adjacents = [v[0] for v in edlist if v[1] == x]
                    adjacents = adjacents + [v[1] for v in edlist if v[0] == x]
                    adjacents = list(set(adjacents))
                    print "\t", titleList[x] + " [" + classList[x] + "]\n" + summaryList[x] + "\n"
                    print "# Adjacents: #"
                    for nt in adjacents:
                        print "\t", titleList[nt] + " [" + classList[nt] + "]\n" + summaryList[nt] + "\n"
                    if(len(adjacents) > 1):
                        print "#Intersection: #"
                        print getIntersectionTexts(adjacents)
                else:
                    index = []
                    for ii in range(sel_list0.GetNumberOfTuples()):
                        x = sel_list0.GetValue(ii)
                        print "\t", titleList[x] + " [" + classList[x] + "]\n" + summaryList[x] + "\n"
                        index.append(x)
                    if(len(index) > 1):
                        print "#Intersection: #"
                        print getIntersectionTexts(index)

            elif node0_field_type == 4:
                for ii in range(sel_list0.GetNumberOfTuples()):
                    x = sel_list0.GetValue(ii)
                    print "\t", str(motivoEdge[x])

        if (sel_list1.GetNumberOfTuples() > 0):
            printFieldType(node1_field_type)
            if node1_field_type == 3:
                if(sel_list1.GetNumberOfTuples() == 1):
                    x = sel_list1.GetValue(0)
                    edlist = [e for e in edgesList if weightList[edgesList.index(e)] < 2.2]
                    adjacents = [v[0] for v in edlist if v[1] == x]
                    adjacents = adjacents + [v[1] for v in edlist if v[0] == x]
                    adjacents = list(set(adjacents))
                    print "\t", titleList[x] + " [" + classList[x] + "]\n" + summaryList[x] + "\n"
                    print "# Adjacents: #"
                    for nt in adjacents:
                        print "\t", titleList[nt] + " [" + classList[nt] + "]\n" + summaryList[nt] + "\n"
                    if(len(adjacents) > 1):
                        print "#Intersection: #"
                        print getIntersectionTexts(adjacents)
                else:
                    index = []
                    for ii in range(sel_list1.GetNumberOfTuples()):
                        x = sel_list1.GetValue(ii)
                        print "\t", titleList[x] + " [" + classList[x] + "]\n" + summaryList[x] + "\n"
                        index.append(x)
                    if(len(index) > 1):
                        print "#Intersection: #"
                        print getIntersectionTexts(index)

            elif node1_field_type == 4:
                for ii in range(sel_list1.GetNumberOfTuples()):
                    x = sel_list1.GetValue(ii)
                    print "\t", str(motivoEdge[x])
        else:
            if(graphLayoutView.GetColorVertices()):
                graphLayoutView.SetColorVertices(False)
            else:
                graphLayoutView.SetColorVertices(True)


        print "- - -\n\n"
    #	GetAdjacentVertices (vtkIdType v, vtkAdjacentVertexIterator *it)
    def printFieldType(field_type):
        if field_type == 3:
            print "Vertices Selected:"
        elif field_type == 4:
            print "Edges Selected:"
        else:
            print "Unknown type:"

    rep = graphLayoutView.GetRepresentation(0)

    # The vtkRenderedGraphRepresentation should already have a vtkAnnotationLink,
    # so we just want to grab it and add an observer with our callback function attached
    link = rep.GetAnnotationLink()
    link.AddObserver("AnnotationChangedEvent", selectionCallback)

    #tempo.append(strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()))
    #print tempo
    print "Terminou"
    print strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())
    print "sozinho: " + str(porsozinho)
    print "otimo: " + str(porotimo)
    print "texto curto: " + str(portextocurto)
    print "vizinhanca: " + str(porvizinho)
    graphLayoutView.GetInteractor().Start()
